{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from newsfeatures import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##feedlist=['https://www.sciencedaily.com/rss/top/science.xml',\n",
    "          'https://www.sciencedaily.com/rss/top/health.xml'\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ok: https://www.sciencedaily.com/rss/top/science.xml\n",
      "ok: https://www.sciencedaily.com/rss/top/health.xml\n"
     ]
    }
   ],
   "source": [
    "for feed in feedlist:\n",
    "    try:\n",
    "        d=feedparser.parse(feed)\n",
    "        print ('ok:' ,feed)\n",
    "    except:\n",
    "        print ('can not parse :',feed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60 news!\n",
      "https://www.sciencedaily.com/rss/top/science.xml   Done!\n",
      "60  news altogether!\n",
      "60 news!\n",
      "https://www.sciencedaily.com/rss/top/health.xml   Done!\n",
      "120  news altogether!\n"
     ]
    }
   ],
   "source": [
    "allw, artw, artt=getarticlewords()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordmatrix,wordvec = makematrix(allw,artw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['some',\n",
       " 'severe',\n",
       " 'covid',\n",
       " 'cases',\n",
       " 'linked',\n",
       " 'genetic',\n",
       " 'antibodies',\n",
       " 'patients',\n",
       " 'other',\n",
       " 'have']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordvec[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'some': 11,\n",
       " 'severe': 7,\n",
       " 'covid': 55,\n",
       " 'cases': 4,\n",
       " 'linked': 6,\n",
       " 'genetic': 8,\n",
       " 'mutations': 3,\n",
       " 'antibodies': 5,\n",
       " 'that': 94,\n",
       " 'attack': 2,\n",
       " 'bodytwo': 1,\n",
       " 'studies': 2,\n",
       " 'offer': 3,\n",
       " 'explanation': 1,\n",
       " 'variable': 1,\n",
       " 'subset': 1,\n",
       " 'patients': 13,\n",
       " 'immunity': 1,\n",
       " 'genes': 3,\n",
       " 'other': 7,\n",
       " 'have': 56,\n",
       " 'auto': 1,\n",
       " 'target': 1,\n",
       " 'same': 1,\n",
       " 'components': 1,\n",
       " 'immune': 10,\n",
       " 'system': 11,\n",
       " 'both': 3,\n",
       " 'circumstances': 1,\n",
       " 'could': 17,\n",
       " 'contribute': 1,\n",
       " 'forms': 3,\n",
       " 'disease': 19,\n",
       " 'comet': 1,\n",
       " 'discovered': 10,\n",
       " 'northern': 2,\n",
       " 'lightscomet': 1,\n",
       " 'churyumov': 1,\n",
       " 'gerasimenko': 1,\n",
       " 'ultraviolet': 3,\n",
       " 'aurora': 1,\n",
       " 'data': 8,\n",
       " 'reveal': 2,\n",
       " 'first': 10,\n",
       " 'time': 6,\n",
       " 'such': 6,\n",
       " 'electromagnetic': 1,\n",
       " 'emissions': 3,\n",
       " 'been': 13,\n",
       " 'documented': 1,\n",
       " 'celestial': 1,\n",
       " 'object': 1,\n",
       " 'than': 12,\n",
       " 'planet': 7,\n",
       " 'moon': 3,\n",
       " 'ancient': 9,\n",
       " 'volcanoes': 1,\n",
       " 'once': 3,\n",
       " 'boosted': 1,\n",
       " 'ocean': 2,\n",
       " 'carbon': 7,\n",
       " 'humans': 2,\n",
       " 'outpacing': 1,\n",
       " 'thema': 1,\n",
       " 'study': 56,\n",
       " 'period': 2,\n",
       " 'considered': 2,\n",
       " 'closest': 1,\n",
       " 'natural': 1,\n",
       " 'analog': 1,\n",
       " 'modern': 2,\n",
       " 'human': 14,\n",
       " 'found': 17,\n",
       " 'massive': 4,\n",
       " 'volcanism': 1,\n",
       " 'sent': 1,\n",
       " 'great': 2,\n",
       " 'waves': 2,\n",
       " 'into': 6,\n",
       " 'oceans': 2,\n",
       " 'over': 4,\n",
       " 'thousands': 1,\n",
       " 'years': 18,\n",
       " 'nature': 1,\n",
       " 'come': 3,\n",
       " 'close': 1,\n",
       " 'matching': 1,\n",
       " 'what': 3,\n",
       " 'doing': 1,\n",
       " 'today': 2,\n",
       " 'possible': 3,\n",
       " 'marker': 1,\n",
       " 'life': 8,\n",
       " 'spotted': 1,\n",
       " 'venusastronomers': 1,\n",
       " 'rare': 2,\n",
       " 'molecule': 1,\n",
       " 'phosphine': 2,\n",
       " 'clouds': 2,\n",
       " 'venus': 2,\n",
       " 'earth': 18,\n",
       " 'this': 11,\n",
       " 'only': 7,\n",
       " 'made': 6,\n",
       " 'industrially': 1,\n",
       " 'microbes': 4,\n",
       " 'thrive': 1,\n",
       " 'oxygen': 2,\n",
       " 'free': 4,\n",
       " 'environments': 1,\n",
       " 'astronomers': 2,\n",
       " 'speculated': 1,\n",
       " 'decades': 1,\n",
       " 'high': 11,\n",
       " 'home': 2,\n",
       " 'floating': 1,\n",
       " 'scorching': 1,\n",
       " 'surface': 4,\n",
       " 'needing': 1,\n",
       " 'tolerate': 1,\n",
       " 'very': 3,\n",
       " 'acidity': 1,\n",
       " 'detection': 2,\n",
       " 'point': 2,\n",
       " 'extra': 1,\n",
       " 'terrestrial': 1,\n",
       " 'aerial': 1,\n",
       " 'hubble': 5,\n",
       " 'suggests': 6,\n",
       " 'there': 8,\n",
       " 'ingredient': 2,\n",
       " 'missing': 3,\n",
       " 'from': 34,\n",
       " 'current': 2,\n",
       " 'dark': 4,\n",
       " 'matter': 4,\n",
       " 'theoriesrecent': 1,\n",
       " 'observations': 4,\n",
       " 'something': 1,\n",
       " 'theories': 2,\n",
       " 'behaves': 1,\n",
       " 'explain': 4,\n",
       " 'researchers': 27,\n",
       " 'uncovered': 4,\n",
       " 'unexpected': 2,\n",
       " 'discrepancy': 1,\n",
       " 'between': 9,\n",
       " 'concentrations': 1,\n",
       " 'sample': 1,\n",
       " 'galaxy': 8,\n",
       " 'clusters': 2,\n",
       " 'theoretical': 3,\n",
       " 'computer': 3,\n",
       " 'simulations': 2,\n",
       " 'should': 2,\n",
       " 'distributed': 1,\n",
       " 'fidelity': 2,\n",
       " 'record': 4,\n",
       " 'climate': 8,\n",
       " 'history': 1,\n",
       " 'puts': 1,\n",
       " 'changes': 7,\n",
       " 'contextscientists': 1,\n",
       " 'compiled': 1,\n",
       " 'continuous': 1,\n",
       " 'variations': 2,\n",
       " 'extending': 1,\n",
       " 'million': 3,\n",
       " 'past': 3,\n",
       " 'reveals': 6,\n",
       " 'four': 2,\n",
       " 'distinctive': 1,\n",
       " 'states': 9,\n",
       " 'which': 13,\n",
       " 'dubbed': 1,\n",
       " 'hothouse': 1,\n",
       " 'warmhouse': 1,\n",
       " 'coolhouse': 1,\n",
       " 'icehouse': 1,\n",
       " 'these': 8,\n",
       " 'major': 4,\n",
       " 'persisted': 1,\n",
       " 'millions': 2,\n",
       " 'sometimes': 1,\n",
       " 'tens': 1,\n",
       " 'within': 3,\n",
       " 'each': 3,\n",
       " 'shows': 12,\n",
       " 'rhythmic': 1,\n",
       " 'corresponding': 1,\n",
       " 'orbit': 1,\n",
       " 'around': 7,\n",
       " 'coronavirus': 11,\n",
       " 'took': 1,\n",
       " 'hold': 3,\n",
       " 'north': 1,\n",
       " 'america': 2,\n",
       " 'europeearly': 1,\n",
       " 'interventions': 1,\n",
       " 'were': 13,\n",
       " 'effective': 8,\n",
       " 'stamping': 1,\n",
       " 'infections': 2,\n",
       " 'before': 8,\n",
       " 'they': 9,\n",
       " 'spread': 3,\n",
       " 'according': 13,\n",
       " 'combining': 2,\n",
       " 'virus': 11,\n",
       " 'genomics': 1,\n",
       " 'with': 40,\n",
       " 'epidemiologic': 1,\n",
       " 'travel': 1,\n",
       " 'records': 2,\n",
       " 'research': 14,\n",
       " 'united': 5,\n",
       " 'europe': 4,\n",
       " 'sustained': 1,\n",
       " 'transmission': 3,\n",
       " 'networks': 3,\n",
       " 'became': 1,\n",
       " 'established': 1,\n",
       " 'after': 10,\n",
       " 'separate': 1,\n",
       " 'introductions': 1,\n",
       " 'went': 2,\n",
       " 'undetected': 1,\n",
       " 'links': 1,\n",
       " 'strict': 2,\n",
       " 'social': 6,\n",
       " 'distancing': 2,\n",
       " 'much': 4,\n",
       " 'lower': 3,\n",
       " 'chance': 1,\n",
       " 'infectionusing': 1,\n",
       " 'public': 1,\n",
       " 'transportation': 1,\n",
       " 'visiting': 1,\n",
       " 'place': 3,\n",
       " 'worship': 1,\n",
       " 'otherwise': 1,\n",
       " 'traveling': 1,\n",
       " 'associated': 10,\n",
       " 'significantly': 2,\n",
       " 'higher': 8,\n",
       " 'likelihood': 2,\n",
       " 'testing': 3,\n",
       " 'positive': 6,\n",
       " 'sars': 21,\n",
       " 'while': 4,\n",
       " 'practicing': 1,\n",
       " 'markedly': 1,\n",
       " 'unconscious': 1,\n",
       " 'learning': 2,\n",
       " 'underlies': 1,\n",
       " 'belief': 1,\n",
       " 'suggestsindividuals': 1,\n",
       " 'unconsciously': 1,\n",
       " 'predict': 2,\n",
       " 'complex': 2,\n",
       " 'patterns': 3,\n",
       " 'ability': 3,\n",
       " 'called': 4,\n",
       " 'implicit': 1,\n",
       " 'pattern': 1,\n",
       " 'likely': 6,\n",
       " 'stronger': 1,\n",
       " 'beliefs': 2,\n",
       " 'creates': 1,\n",
       " 'events': 1,\n",
       " 'universe': 6,\n",
       " 'neuroscientists': 5,\n",
       " 'process': 6,\n",
       " 'might': 4,\n",
       " 'organic': 2,\n",
       " 'moleculesnew': 1,\n",
       " 'identifies': 2,\n",
       " 'producing': 2,\n",
       " 'molecules': 6,\n",
       " 'about': 7,\n",
       " 'billion': 5,\n",
       " 'origin': 1,\n",
       " 'similar': 2,\n",
       " 'occurred': 1,\n",
       " 'underwater': 1,\n",
       " 'hydrothermal': 1,\n",
       " 'vents': 1,\n",
       " 'also': 8,\n",
       " 'relevance': 1,\n",
       " 'search': 3,\n",
       " 'elsewhere': 1,\n",
       " 'true': 1,\n",
       " 'size': 2,\n",
       " 'prehistoric': 2,\n",
       " 'mega': 1,\n",
       " 'shark': 2,\n",
       " 'finally': 1,\n",
       " 'revealeda': 1,\n",
       " 'revealed': 7,\n",
       " 'legendary': 1,\n",
       " 'giant': 3,\n",
       " 'megalodon': 1,\n",
       " 'including': 2,\n",
       " 'fins': 1,\n",
       " 'large': 5,\n",
       " 'adult': 2,\n",
       " 'rusted': 1,\n",
       " 'billions': 2,\n",
       " 'surprise': 1,\n",
       " 'many': 8,\n",
       " 'planetary': 1,\n",
       " 'scientists': 10,\n",
       " 'oxidized': 1,\n",
       " 'iron': 1,\n",
       " 'mineral': 2,\n",
       " 'hematite': 1,\n",
       " 'latitudes': 1,\n",
       " 'bang': 1,\n",
       " 'ligo': 1,\n",
       " 'virgo': 1,\n",
       " 'detectors': 1,\n",
       " 'signals': 1,\n",
       " 'most': 7,\n",
       " 'gravitational': 2,\n",
       " 'wave': 3,\n",
       " 'source': 3,\n",
       " 'yetresearchers': 1,\n",
       " 'detected': 1,\n",
       " 'signal': 1,\n",
       " 'black': 5,\n",
       " 'hole': 2,\n",
       " 'merger': 2,\n",
       " 'observed': 3,\n",
       " 'product': 1,\n",
       " 'clear': 1,\n",
       " 'intermediate': 2,\n",
       " 'mass': 4,\n",
       " 'times': 1,\n",
       " 'largest': 2,\n",
       " 'solar': 7,\n",
       " 'telescope': 3,\n",
       " 'gregor': 2,\n",
       " 'unveils': 1,\n",
       " 'magnetic': 3,\n",
       " 'details': 2,\n",
       " 'sungregor': 1,\n",
       " 'obtained': 1,\n",
       " 'unprecedented': 1,\n",
       " 'images': 2,\n",
       " 'fine': 2,\n",
       " 'structure': 2,\n",
       " 'following': 2,\n",
       " 'redesign': 1,\n",
       " 'optics': 1,\n",
       " 'resolution': 2,\n",
       " 'discover': 2,\n",
       " 'specific': 6,\n",
       " 'brain': 17,\n",
       " 'circuit': 1,\n",
       " 'damaged': 1,\n",
       " 'isolation': 2,\n",
       " 'during': 16,\n",
       " 'childhoodresearchers': 1,\n",
       " 'identified': 4,\n",
       " 'populations': 2,\n",
       " 'cells': 21,\n",
       " 'prefrontal': 1,\n",
       " 'cortex': 2,\n",
       " 'part': 4,\n",
       " 'regulates': 2,\n",
       " 'behavior': 4,\n",
       " 'required': 1,\n",
       " 'normal': 4,\n",
       " 'sociability': 1,\n",
       " 'adulthood': 1,\n",
       " 'profoundly': 1,\n",
       " 'vulnerable': 4,\n",
       " 'juvenile': 1,\n",
       " 'mice': 7,\n",
       " 'genetics': 1,\n",
       " 'tree': 3,\n",
       " 'lifebaobab': 1,\n",
       " 'trees': 1,\n",
       " 'live': 3,\n",
       " 'more': 17,\n",
       " 'thousand': 2,\n",
       " 'provide': 3,\n",
       " 'food': 1,\n",
       " 'livestock': 1,\n",
       " 'fodder': 1,\n",
       " 'medicinal': 1,\n",
       " 'compounds': 1,\n",
       " 'materials': 3,\n",
       " 'counted': 1,\n",
       " 'significant': 2,\n",
       " 'chromosomes': 3,\n",
       " 'information': 2,\n",
       " 'critical': 4,\n",
       " 'conservation': 1,\n",
       " 'agricultural': 1,\n",
       " 'improvement': 1,\n",
       " 'further': 3,\n",
       " 'work': 3,\n",
       " 'maps': 1,\n",
       " 'halo': 2,\n",
       " 'andromeda': 2,\n",
       " 'galaxyin': 1,\n",
       " 'landmark': 1,\n",
       " 'using': 7,\n",
       " 'nasa': 5,\n",
       " 'space': 4,\n",
       " 'mapped': 2,\n",
       " 'immense': 1,\n",
       " 'enveloping': 1,\n",
       " 'nearest': 1,\n",
       " 'galactic': 3,\n",
       " 'neighbor': 1,\n",
       " 'meteorite': 2,\n",
       " 'since': 2,\n",
       " 'formeda': 1,\n",
       " 'finds': 9,\n",
       " 'water': 7,\n",
       " 'present': 1,\n",
       " 'inner': 1,\n",
       " 'formed': 1,\n",
       " 'instead': 1,\n",
       " 'reaching': 1,\n",
       " 'comets': 1,\n",
       " 'asteroids': 1,\n",
       " 'delivering': 1,\n",
       " 'findings': 4,\n",
       " 'suggest': 4,\n",
       " 'always': 1,\n",
       " 'artificial': 3,\n",
       " 'pancreas': 2,\n",
       " 'effectively': 3,\n",
       " 'controls': 1,\n",
       " 'type': 4,\n",
       " 'diabetes': 6,\n",
       " 'children': 8,\n",
       " 'clinical': 3,\n",
       " 'trial': 1,\n",
       " 'pediatric': 1,\n",
       " 'centers': 1,\n",
       " 'automatically': 1,\n",
       " 'monitors': 1,\n",
       " 'blood': 5,\n",
       " 'glucose': 2,\n",
       " 'levels': 7,\n",
       " 'safe': 2,\n",
       " 'managing': 2,\n",
       " 'young': 5,\n",
       " 'strikes': 1,\n",
       " 'create': 2,\n",
       " 'form': 3,\n",
       " 'silicanew': 1,\n",
       " 'examining': 2,\n",
       " 'crystal': 1,\n",
       " 'silica': 1,\n",
       " 'quartz': 1,\n",
       " 'under': 2,\n",
       " 'shock': 1,\n",
       " 'compression': 1,\n",
       " 'challenging': 1,\n",
       " 'longstanding': 1,\n",
       " 'assumptions': 1,\n",
       " 'ubiquitous': 1,\n",
       " 'material': 1,\n",
       " 'transplanted': 1,\n",
       " 'brown': 3,\n",
       " 'like': 8,\n",
       " 'promise': 3,\n",
       " 'obesity': 2,\n",
       " 'diabetesa': 1,\n",
       " 'potential': 7,\n",
       " 'therapy': 2,\n",
       " 'would': 3,\n",
       " 'transplant': 1,\n",
       " 'humble': 1,\n",
       " 'white': 3,\n",
       " 'genetically': 1,\n",
       " 'modified': 1,\n",
       " 'crispr': 1,\n",
       " 'become': 2,\n",
       " 'heat': 1,\n",
       " 'generating': 1,\n",
       " 'female': 3,\n",
       " 'resilience': 1,\n",
       " 'alzheimer': 8,\n",
       " 'swomen': 1,\n",
       " 'longer': 1,\n",
       " 'because': 1,\n",
       " 'their': 8,\n",
       " 'give': 1,\n",
       " 'them': 1,\n",
       " 'protection': 2,\n",
       " 'ravages': 1,\n",
       " 'women': 5,\n",
       " 'doses': 1,\n",
       " 'gene': 2,\n",
       " 'exists': 1,\n",
       " 'chromosome': 1,\n",
       " 'people': 18,\n",
       " 'male': 2,\n",
       " 'especially': 1,\n",
       " 'potent': 1,\n",
       " 'variant': 2,\n",
       " 'long': 13,\n",
       " 'term': 7,\n",
       " 'older': 1,\n",
       " 'whom': 1,\n",
       " 'already': 1,\n",
       " 'mild': 1,\n",
       " 'cognitive': 6,\n",
       " 'impairment': 2,\n",
       " 'showed': 2,\n",
       " 'copies': 1,\n",
       " 'progressed': 1,\n",
       " 'slowly': 2,\n",
       " 'toward': 1,\n",
       " 'cosmic': 4,\n",
       " 'rays': 3,\n",
       " 'soon': 4,\n",
       " 'stymie': 2,\n",
       " 'quantum': 4,\n",
       " 'computinginfinitesimally': 1,\n",
       " 'radiation': 2,\n",
       " 'incoming': 1,\n",
       " 'progress': 1,\n",
       " 'computing': 1,\n",
       " 'fatigue': 2,\n",
       " 'pinpoints': 1,\n",
       " 'regions': 3,\n",
       " 'control': 3,\n",
       " 'itusing': 1,\n",
       " 'scans': 1,\n",
       " 'modeling': 1,\n",
       " 'pinpointed': 1,\n",
       " 'areas': 2,\n",
       " 'regulate': 2,\n",
       " 'efforts': 2,\n",
       " 'deal': 1,\n",
       " 'paradox': 2,\n",
       " 'resolved': 2,\n",
       " 'dancenew': 1,\n",
       " 'light': 6,\n",
       " 'shed': 1,\n",
       " 'mysterious': 1,\n",
       " 'standing': 1,\n",
       " 'conundrum': 1,\n",
       " 'heart': 4,\n",
       " 'offers': 2,\n",
       " 'solution': 2,\n",
       " 'whereby': 1,\n",
       " 'different': 4,\n",
       " 'produce': 1,\n",
       " 'contradictory': 1,\n",
       " 'estimates': 2,\n",
       " 'motion': 2,\n",
       " 'central': 2,\n",
       " 'milky': 3,\n",
       " 'stop': 3,\n",
       " 'start': 1,\n",
       " 'radio': 1,\n",
       " 'jetsa': 1,\n",
       " 'vlba': 1,\n",
       " 'image': 1,\n",
       " 'emitted': 1,\n",
       " 'core': 1,\n",
       " 'active': 4,\n",
       " 'revealing': 1,\n",
       " 'activity': 3,\n",
       " 'stopped': 1,\n",
       " 'then': 2,\n",
       " 'restarted': 1,\n",
       " 'decade': 1,\n",
       " 'star': 3,\n",
       " 'explosions': 1,\n",
       " 'deep': 3,\n",
       " 'sedimentsa': 1,\n",
       " 'mystery': 3,\n",
       " 'surrounding': 1,\n",
       " 'unfolding': 1,\n",
       " 'thanks': 2,\n",
       " 'evidence': 5,\n",
       " 'supernovae': 2,\n",
       " 'sediments': 1,\n",
       " 'optical': 2,\n",
       " 'illusions': 2,\n",
       " 'explained': 1,\n",
       " 'eyeswhy': 1,\n",
       " 'perceive': 1,\n",
       " 'static': 1,\n",
       " 'mystified': 1,\n",
       " 'those': 4,\n",
       " 'view': 1,\n",
       " 'tried': 1,\n",
       " 'phenomenon': 1,\n",
       " 'answers': 2,\n",
       " 'eyes': 1,\n",
       " 'flies': 1,\n",
       " 'genomic': 1,\n",
       " 'analysis': 3,\n",
       " 'animal': 2,\n",
       " 'species': 3,\n",
       " 'infectionanalysis': 1,\n",
       " 'ace2': 2,\n",
       " 'main': 1,\n",
       " 'receptor': 2,\n",
       " 'uses': 3,\n",
       " 'bind': 2,\n",
       " 'enter': 1,\n",
       " 'across': 2,\n",
       " 'vertebrate': 1,\n",
       " 'potentially': 2,\n",
       " 'susceptible': 1,\n",
       " 'infection': 8,\n",
       " 'novel': 3,\n",
       " 'include': 2,\n",
       " 'number': 5,\n",
       " 'endangered': 1,\n",
       " 'threatened': 1,\n",
       " 'notably': 1,\n",
       " 'apes': 1,\n",
       " 'world': 6,\n",
       " 'primates': 1,\n",
       " 'hosts': 1,\n",
       " 'models': 1,\n",
       " 'exploding': 1,\n",
       " 'stars': 2,\n",
       " 'caused': 4,\n",
       " 'extinction': 3,\n",
       " 'showskiller': 1,\n",
       " 'nearby': 1,\n",
       " 'culprit': 2,\n",
       " 'behind': 2,\n",
       " 'least': 3,\n",
       " 'event': 1,\n",
       " 'said': 1,\n",
       " 'finding': 4,\n",
       " 'certain': 2,\n",
       " 'radioactive': 1,\n",
       " 'isotopes': 1,\n",
       " 'rock': 1,\n",
       " 'confirm': 1,\n",
       " 'scenario': 1,\n",
       " 'track': 1,\n",
       " 'splitting': 1,\n",
       " 'dent': 1,\n",
       " 'fieldearth': 1,\n",
       " 'field': 4,\n",
       " 'acts': 2,\n",
       " 'protective': 1,\n",
       " 'shield': 1,\n",
       " 'repelling': 2,\n",
       " 'trapping': 1,\n",
       " 'charged': 1,\n",
       " 'particles': 3,\n",
       " 'south': 2,\n",
       " 'southern': 1,\n",
       " 'atlantic': 2,\n",
       " 'unusually': 1,\n",
       " 'weak': 1,\n",
       " 'spot': 1,\n",
       " 'anomaly': 1,\n",
       " 'allows': 1,\n",
       " 'closer': 1,\n",
       " 'loss': 2,\n",
       " 'enzyme': 3,\n",
       " 'boosts': 1,\n",
       " 'metabolism': 2,\n",
       " 'exercise': 3,\n",
       " 'endurance': 2,\n",
       " 'miceblocking': 1,\n",
       " 'regulating': 1,\n",
       " 'muscles': 2,\n",
       " 'leads': 1,\n",
       " 'increased': 4,\n",
       " 'capacity': 1,\n",
       " 'results': 3,\n",
       " 'microbiome': 4,\n",
       " 'role': 7,\n",
       " 'attacking': 1,\n",
       " 'cancerous': 2,\n",
       " 'tumorsresearchers': 1,\n",
       " 'bacteria': 3,\n",
       " 'help': 8,\n",
       " 'battle': 1,\n",
       " 'tumors': 1,\n",
       " 'discovery': 1,\n",
       " 'understanding': 3,\n",
       " 'immunotherapy': 2,\n",
       " 'treatment': 4,\n",
       " 'cancer': 5,\n",
       " 'helps': 5,\n",
       " 'amplify': 1,\n",
       " 'response': 3,\n",
       " 'works': 1,\n",
       " 'others': 1,\n",
       " 'show': 2,\n",
       " 'microbial': 1,\n",
       " 'recognize': 1,\n",
       " 'three': 2,\n",
       " 'seasonal': 1,\n",
       " 'vaccinations': 1,\n",
       " 'stick': 1,\n",
       " 'bone': 2,\n",
       " 'marrowseasonal': 1,\n",
       " 'vaccination': 2,\n",
       " 'does': 3,\n",
       " 'increase': 4,\n",
       " 'antibody': 3,\n",
       " 'marrow': 1,\n",
       " 'however': 3,\n",
       " 'newly': 2,\n",
       " 'generated': 1,\n",
       " 'lost': 3,\n",
       " 'year': 1,\n",
       " 'when': 9,\n",
       " 'smiling': 2,\n",
       " 'whole': 1,\n",
       " 'really': 1,\n",
       " 'smile': 2,\n",
       " 'youfrom': 1,\n",
       " 'sinatra': 1,\n",
       " 'katy': 1,\n",
       " 'perry': 1,\n",
       " 'celebrities': 1,\n",
       " 'sung': 1,\n",
       " 'power': 1,\n",
       " 'picks': 1,\n",
       " 'your': 3,\n",
       " 'outlook': 1,\n",
       " 'generally': 2,\n",
       " 'makes': 2,\n",
       " 'feel': 1,\n",
       " 'better': 3,\n",
       " 'smoke': 1,\n",
       " 'mirrors': 1,\n",
       " 'scientific': 3,\n",
       " 'backing': 1,\n",
       " 'claim': 2,\n",
       " 'groundbreaking': 2,\n",
       " 'confirms': 1,\n",
       " 'trick': 1,\n",
       " 'mind': 4,\n",
       " 'being': 2,\n",
       " 'simply': 1,\n",
       " 'moving': 1,\n",
       " 'facial': 1,\n",
       " 'warming': 2,\n",
       " 'greenland': 2,\n",
       " 'sheet': 2,\n",
       " 'passes': 1,\n",
       " 'returnnearly': 1,\n",
       " 'satellite': 2,\n",
       " 'glaciers': 1,\n",
       " 'island': 1,\n",
       " 'shrunk': 1,\n",
       " 'even': 5,\n",
       " 'global': 3,\n",
       " 'continue': 2,\n",
       " 'shrinking': 1,\n",
       " 'genomes': 1,\n",
       " 'woolly': 3,\n",
       " 'rhinos': 2,\n",
       " 'extinct': 2,\n",
       " 'change': 4,\n",
       " 'overhuntingalthough': 1,\n",
       " 'overhunting': 1,\n",
       " 'demise': 1,\n",
       " 'megafauna': 1,\n",
       " 'last': 3,\n",
       " 'rhinoceros': 1,\n",
       " 'sequencing': 1,\n",
       " 'population': 1,\n",
       " 'remained': 1,\n",
       " 'stable': 1,\n",
       " 'diverse': 1,\n",
       " 'until': 2,\n",
       " 'disappeared': 1,\n",
       " 'siberia': 1,\n",
       " 'temperatures': 2,\n",
       " 'rose': 1,\n",
       " 'make': 2,\n",
       " 'green': 2,\n",
       " 'chemistry': 2,\n",
       " 'advance': 2,\n",
       " 'catalyst': 1,\n",
       " 'reduction': 3,\n",
       " 'dioxideresearchers': 1,\n",
       " 'pursuit': 1,\n",
       " 'converting': 2,\n",
       " 'greenhouse': 1,\n",
       " 'dioxide': 5,\n",
       " 'reusable': 1,\n",
       " 'electrochemical': 1,\n",
       " 'yoga': 2,\n",
       " 'shown': 5,\n",
       " 'improve': 4,\n",
       " 'anxiety': 5,\n",
       " 'showsa': 1,\n",
       " 'improves': 3,\n",
       " 'symptoms': 4,\n",
       " 'generalized': 1,\n",
       " 'disorder': 3,\n",
       " 'suggesting': 2,\n",
       " 'popular': 1,\n",
       " 'practice': 1,\n",
       " 'helpful': 1,\n",
       " 'treating': 2,\n",
       " 'error': 2,\n",
       " 'correcting': 2,\n",
       " 'catphysicists': 1,\n",
       " 'developed': 6,\n",
       " 'device': 2,\n",
       " 'combines': 1,\n",
       " 'schrödinger': 1,\n",
       " 'concept': 3,\n",
       " 'superposition': 1,\n",
       " 'physical': 2,\n",
       " 'existing': 1,\n",
       " 'trickiest': 1,\n",
       " 'errors': 1,\n",
       " 'computation': 1,\n",
       " 'extremely': 4,\n",
       " 'look': 2,\n",
       " 'alikeastronomers': 1,\n",
       " 'atacama': 1,\n",
       " 'millimeter': 1,\n",
       " 'submillimeter': 1,\n",
       " 'array': 1,\n",
       " 'alma': 3,\n",
       " 'distant': 2,\n",
       " 'therefore': 1,\n",
       " 'looks': 1,\n",
       " 'surprisingly': 3,\n",
       " 'away': 1,\n",
       " 'taken': 1,\n",
       " 'reach': 1,\n",
       " 'just': 2,\n",
       " 'unchaotic': 1,\n",
       " 'contradicting': 1,\n",
       " 'galaxies': 2,\n",
       " 'early': 5,\n",
       " 'turbulent': 1,\n",
       " 'unstable': 1,\n",
       " 'dwarf': 2,\n",
       " 'supernova': 5,\n",
       " 'physicist': 1,\n",
       " 'calculates': 1,\n",
       " 'ever': 1,\n",
       " 'will': 2,\n",
       " 'happennew': 1,\n",
       " 'dwarfs': 1,\n",
       " 'explode': 1,\n",
       " 'future': 3,\n",
       " 'everything': 1,\n",
       " 'else': 1,\n",
       " 'died': 2,\n",
       " 'gone': 1,\n",
       " 'quiet': 1,\n",
       " 'inexpensive': 1,\n",
       " 'accessible': 2,\n",
       " 'provides': 1,\n",
       " 'visual': 2,\n",
       " 'proof': 4,\n",
       " 'masks': 6,\n",
       " 'block': 2,\n",
       " 'dropletsin': 1,\n",
       " 'report': 3,\n",
       " 'simple': 1,\n",
       " 'cost': 2,\n",
       " 'technique': 1,\n",
       " 'provided': 2,\n",
       " 'face': 4,\n",
       " 'reducing': 2,\n",
       " 'droplet': 1,\n",
       " 'wear': 1,\n",
       " 'best': 4,\n",
       " 'coverings': 3,\n",
       " 'without': 4,\n",
       " 'valves': 1,\n",
       " 'hospital': 3,\n",
       " 'grade': 1,\n",
       " 'used': 3,\n",
       " 'front': 1,\n",
       " 'line': 1,\n",
       " 'health': 5,\n",
       " 'care': 8,\n",
       " 'workers': 1,\n",
       " 'surgical': 2,\n",
       " 'polypropylene': 1,\n",
       " 'performed': 2,\n",
       " 'well': 4,\n",
       " 'hand': 2,\n",
       " 'cotton': 1,\n",
       " 'good': 3,\n",
       " 'coverage': 1,\n",
       " 'eliminating': 1,\n",
       " 'substantial': 1,\n",
       " 'amount': 1,\n",
       " 'spray': 1,\n",
       " 'speech': 1,\n",
       " 'bandanas': 1,\n",
       " 'neck': 1,\n",
       " 'fleeces': 1,\n",
       " 'balaclavas': 1,\n",
       " 'didn': 1,\n",
       " 'droplets': 1,\n",
       " 'solved': 1,\n",
       " 'bright': 1,\n",
       " 'ceres': 2,\n",
       " 'salty': 1,\n",
       " 'belowdata': 1,\n",
       " 'recent': 1,\n",
       " 'dawn': 1,\n",
       " 'mission': 3,\n",
       " 'unresolved': 1,\n",
       " 'questions': 1,\n",
       " 'liquid': 1,\n",
       " 'inside': 3,\n",
       " 'geologically': 1,\n",
       " 'supports': 2,\n",
       " 'complete': 2,\n",
       " 'arctic': 2,\n",
       " '2035a': 1,\n",
       " 'predictions': 1,\n",
       " '2035': 1,\n",
       " 'photosynthetic': 3,\n",
       " 'hacks': 1,\n",
       " 'boost': 2,\n",
       " 'crop': 1,\n",
       " 'yield': 2,\n",
       " 'conserve': 2,\n",
       " 'waterplants': 1,\n",
       " 'factories': 1,\n",
       " 'manufacture': 1,\n",
       " 'parts': 1,\n",
       " 'photosynthesis': 1,\n",
       " 'hindered': 1,\n",
       " 'lack': 1,\n",
       " 'machinery': 2,\n",
       " 'optimize': 1,\n",
       " 'production': 1,\n",
       " 'bottlenecks': 1,\n",
       " 'plant': 1,\n",
       " 'productivity': 1,\n",
       " 'percent': 3,\n",
       " 'real': 1,\n",
       " 'conditions': 3,\n",
       " 'hack': 1,\n",
       " 'mars': 5,\n",
       " 'covered': 1,\n",
       " 'sheets': 1,\n",
       " 'flowing': 2,\n",
       " 'rivers': 3,\n",
       " 'saya': 1,\n",
       " 'valley': 1,\n",
       " 'scarring': 1,\n",
       " 'carved': 1,\n",
       " 'melting': 1,\n",
       " 'beneath': 1,\n",
       " 'glacial': 1,\n",
       " 'previously': 2,\n",
       " 'thought': 1,\n",
       " 'throw': 1,\n",
       " 'cold': 1,\n",
       " 'dominant': 1,\n",
       " 'warm': 1,\n",
       " 'hypothesis': 2,\n",
       " 'postulates': 1,\n",
       " 'rainfall': 1,\n",
       " 'existed': 1,\n",
       " 'underpin': 1,\n",
       " 'covidnew': 1,\n",
       " 'patient': 3,\n",
       " 'analyses': 1,\n",
       " 'overactive': 1,\n",
       " 'complement': 1,\n",
       " 'oldest': 1,\n",
       " 'branches': 1,\n",
       " 'excess': 1,\n",
       " 'clotting': 1,\n",
       " 'astronauts': 2,\n",
       " 'safely': 3,\n",
       " 'splash': 1,\n",
       " 'down': 2,\n",
       " 'commercial': 3,\n",
       " 'crew': 2,\n",
       " 'flight': 2,\n",
       " 'stationtwo': 1,\n",
       " 'splashed': 1,\n",
       " 'gulf': 1,\n",
       " 'mexico': 1,\n",
       " 'sunday': 1,\n",
       " 'commercially': 1,\n",
       " 'built': 2,\n",
       " 'operated': 1,\n",
       " 'american': 2,\n",
       " ...}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0, 0, 0, 0, 1, 2]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordmatrix[1][0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys,os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_path= os.getcwd()\n",
    "\n",
    "\n",
    "current_dir=os.path.dirname(current_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "c3_path=os.path.join(current_dir,'第03章 发现群组')\n",
    "\n",
    "\n",
    "c6_path=os.path.join(current_dir,'第06章 文档过滤')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append(c3_path)\n",
    "sys.path.append(c6_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import docclass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wordmatrixfeatures(x):\n",
    "    return [wordvec[w] for w in range(len(x)) if x[w]>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['some',\n",
       " 'severe',\n",
       " 'covid',\n",
       " 'cases',\n",
       " 'linked',\n",
       " 'genetic',\n",
       " 'antibodies',\n",
       " 'patients',\n",
       " 'other',\n",
       " 'have',\n",
       " 'immune',\n",
       " 'system',\n",
       " 'could',\n",
       " 'disease']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordmatrixfeatures(wordmatrix[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier=docclass.naivebayes(wordmatrixfeatures)\n",
    "classifier.setdb('newstesst.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier.train(wordmatrix[0],'human')\n",
    "classifier.train(wordmatrix[1],'life')\n",
    "classifier.train(wordmatrix[2],'Hubble')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'life'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.classify(wordmatrix[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "clust = clusters.hcluster(wordmatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters.drawdendrogram(clust,artt,jpeg='sciencedaily.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def showarticles(titles,toppatterns,patternnames,out='articles_1.txt'):\n",
    "\toutfile=open(out,'w')  \n",
    "\t\n",
    "\t# Loop over all the articles\n",
    "\tfor j in range(len(titles)):\n",
    "\t\toutfile.write(titles[j]+'\\n')\n",
    "\t\t\n",
    "\t\t# Get the top features for this article and\n",
    "\t\t# reverse sort them\n",
    "\t\ttoppatterns[j].sort()\n",
    "\t\ttoppatterns[j].reverse()\n",
    "\t\t\n",
    "\t\t# Print the top three patterns\n",
    "\t\tfor i in range(3):\n",
    "\t\t\toutfile.write(str(toppatterns[j][i][0])+' '+\n",
    "\t\t\t\t\t\t\t\t\t\tstr(patternnames[toppatterns[j][i][1]])+'\\n')\n",
    "\t\toutfile.write('\\n')\n",
    "\t\t\n",
    "\toutfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nnmf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "v= matrix(wordmatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "617477.3031479744\n",
      "1335.6219449614352\n",
      "1245.338645126424\n",
      "1215.2800600214703\n",
      "1209.9927377984313\n"
     ]
    }
   ],
   "source": [
    "weights,feat = nnmf.factorize(v,pc=20,iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[4.29091794e-003, 8.14662033e-068, 1.21374410e-007, ...,\n",
       "         3.00050965e-003, 3.30268735e-095, 1.81600807e-092],\n",
       "        [6.46473215e-002, 4.49726046e-042, 2.69956271e-003, ...,\n",
       "         4.71372634e-024, 3.08527836e-071, 1.25022669e-016],\n",
       "        [2.01558019e-034, 6.57805606e-034, 9.04158631e-002, ...,\n",
       "         3.52348242e-057, 8.78213450e-003, 2.57554099e-002],\n",
       "        ...,\n",
       "        [8.06140109e-025, 0.00000000e+000, 4.47673523e-010, ...,\n",
       "         5.31384504e-011, 6.45037696e-004, 1.92591964e-033],\n",
       "        [4.08011802e-006, 5.61663406e-023, 1.98495331e-002, ...,\n",
       "         3.14742394e-002, 1.59544869e-002, 1.00202733e-002],\n",
       "        [8.66538544e-003, 3.57490472e-019, 3.64619615e-043, ...,\n",
       "         1.25121950e-002, 1.41891351e-002, 6.17941957e-142]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "topp,pn=showfeatures(weights,feat,artt,wordvec,out='f01.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'some'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordvec[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "showarticles(artt,topp,pn)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
